#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Basic;

use strict;
use LWP::UserAgent;
use HTTP::Request::Common qw(GET);
use HTTP::Cookies;
use LWP::Simple;
use HTTP::Status ();
use File::Path qw(make_path);
 
my %AlreadyScanned = ();
my $AlreadyScanned;

my $D;
my %D = ();

my $ScannedFile = "/home/thecerial/trendcrawler/logfile/alreadyscanned.txt";
my $CookieFile = "/home/thecerial/trendcrawler/logfile/cookies.txt";

# Konstruktor
sub new {
	my ($class) = @_;
	my $self = {
	#	'kontostand' => $inKontostand,
	#	'dispo' => $inDispo
	};
	bless $self, $class;
  make_path '/home/thecerial/trendcrawler/logfile/', {owner=>'thecerial', group=>'users'};
  return $self;
}

sub fetch(){

	my $self	= shift;
	my $url 	= shift;
	return if $url !~ m/^http/ig;
	
	my $max_retry = 30;
	my $sleep = 60;
	
	if (exists $AlreadyScanned{$url} || exists $D{$url} ){
		warn "$self->$url wurde schon gescannt\n";
		return 0;
	}
	
	my $ua = LWP::UserAgent->new;
		
		$ua->timeout(180);
		$ua->max_redirect( 10 );
		
		# Define user agent type
		$ua->agent('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
	
		# Cookies
		$ua->cookie_jar(
			HTTP::Cookies->new(
				file => $CookieFile,
				autosave => 1
			)
		);

	while($max_retry>0){	
		# Request object
		my $req = GET $url;

		# Make the request
		my $res = $ua->request($req);
	
		if ($res->is_error) {
			printf "$self->Fetching $url -> Error Code: [%d] %s\n", $res->code, $res->message;
			$max_retry--;
			sleep $sleep;
		}
		
		if ( $res->is_success ){
			 
			open(W,"+>>$ScannedFile");
				print W "$url\n";
			close W;
					 
			return $res->content;		 
		}
	}
	
	return 0;
	
} # sub fetch(){


sub check(){
	
	### Check sollte den Server anfragen stellen, der den SQL DB enthält
	
	my $max_retry 				= 30;
	my $sleep 					= 30;
	
	my $length_of_randomstring 	= 32; # the length of the random string to generate

	my @chars = ('a'..'z','A'..'Z','0'..'9','_');
	my $random_string = "";
	
	foreach (1..$length_of_randomstring){
		# rand @chars will generate a random 
		# number between 0 and scalar @chars
		$random_string .= $chars[rand @chars];
	}
	
	my $ua = LWP::UserAgent->new;
	$ua->timeout(10);
	
	while($max_retry>0){
		
		my $response = $ua->get("http://www.nookiestar.com/alive.php?id=$random_string");

		if ($response->is_error) {
			printf "Error: [%d] %s\n", $response->code, $response->message;

			# record the timeout
			if ($response->code == HTTP::Status::HTTP_INTERNAL_SERVER_ERROR) {
				# later: socket: phprestart
				$max_retry--;
				print "HTTP 500 -> HTTP_INTERNAL_SERVER_ERROR: waiting $sleep second\n";
				sleep $sleep;
			}
		}
		
		if ( $response->content =~ /online/ig ){
			return 1;
		}
		
		if ( $response->content =~ /offline/ig ){
			$max_retry--;
			print "Mysql Database has gone away: waiting $sleep second\n";
			sleep $sleep;
		}
		
	}

	return 0;
}

sub uniq() {
	my %seen;
	return grep { !$seen{$_}++ } @_;
} # sub uniq() {

sub links(){
	
	my $html 		= shift;
	my $forbidden 	= shift;
	my $base 		= shift;
	my $min_link_lenght = shift;
	my $musthave		= shift;
	
	my $lenght 		= length $base;
	my @links 		= ();
	
	my $LX 			= new HTML::LinkExtractor();
    $LX->parse(\$html);
	
    for my $Link( @{ $LX->links } ) {
           if( $$Link{href} !~ m/$forbidden/ig 
			&& $$Link{href} =~ m/$base/ig 
		 	&& $$Link{href} =~ m/$musthave/ig  # funzt nicht
			&& length($$Link{href})>$lenght+$min_link_lenght ) {
			
          #  print "$scriptname ->Found Link: ". $$Link{href}."\n";
			push @links, $$Link{href};
        }
    }
	
	my @unique = uniq( @links );
	return \@unique;
	
} #sub links(){